# -*- coding: utf-8 -*-

from scrapy.selector import Selector
from scrapy.spider import Spider

from scrapy.utils.response import get_base_url
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as sle
from scrapy.http import Request
from xs.items import *
import os
import json
import re


class QQWenXueSpider(CrawlSpider):
    name = 'qqwenxue'
    allowed_domains = ['qq.com']
    start_urls = [
        'http://book.qq.com/intro.html?bid=464309'
    ]
    # rules = [
    #     Rule(sle(allow=("/50/50046/?$"), unique=True), callback='parse_chapter'),
    #     # Rule(sle(allow=("/50/50046/\d+\.html/?$"), unique=True), callback='parse_1'),
    # ]



    def parse(self, response):
        self.log('Hi, this is an chapter page! %s' % response.url)
        
        url_counts = response.xpath('//*[@id="lastchapters"]/div/dl[1]/dd[1]/a/@href').re('cid=(\d*)')[0]
        bookName = response.xpath('//*[@class="book_info"]/h3/a/text()').extract()[0]

        for count in range(1, int(url_counts)+1):
            item = XsItem()
            item['bookName'] = bookName
            item['chapterId'] = count

            url = 'http://book.qq.com/read/464309/%d' % count
            self.log('hi,it is %s' % url)
            yield Request(url, meta = {'item': item}, callback = self.parse_1)


    def parse_1(self, response):
        # items = []
        # sel = Selector(response)
        # sites = sel.xpath('//div[@class="bdsub"]/dl')
        # items = []

        # for site in sites:
        #     item = XsItem()
        #     item['chapterName'] = site.xpath('dd/h1/text()').extract()
        #     items.append(item)
        self.log('Hi, this is an item page! %s' % response.url)

        # item = XsItem()
        item = response.meta['item']
        data = json.loads(response.body).get('Content')


        item['chapterName'] = re.findall('<h1 nodetype=\"chapterTitle\" class=\"story_title\">(.*)<\/h1>', data)[0]
        item['chapterContent'] = re.findall('<div class=\"text\"><div class=\"bookreadercontent\">(.*)<p style=\"', data)[0]
        return item